{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Linking without deduplication\n",
        "\n",
        "A simple record linkage model using the `link_only` [link type](https://moj-analytical-services.github.io/splink/settings_dict_guide.html#link_type).\n",
        "\n",
        "With `link_only`, only between-dataset record comparisons are generated. No within-dataset record comparisons are created, meaning that the model does not attempt to find within-dataset duplicates.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<a target=\"_blank\" href=\"https://colab.research.google.com/github/moj-analytical-services/splink/blob/master/docs/demos/examples/duckdb/link_only.ipynb\">\n",
        "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/>\n",
        "</a>\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:42.926356Z",
          "iopub.status.busy": "2024-06-07T09:18:42.925982Z",
          "iopub.status.idle": "2024-06-07T09:18:42.943456Z",
          "shell.execute_reply": "2024-06-07T09:18:42.942569Z"
        },
        "tags": [
          "hide_input"
        ]
      },
      "outputs": [],
      "source": [
        "# Uncomment and run this cell if you're running in Google Colab.\n",
        "# !pip install splink"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:42.947959Z",
          "iopub.status.busy": "2024-06-07T09:18:42.947640Z",
          "iopub.status.idle": "2024-06-07T09:18:44.652788Z",
          "shell.execute_reply": "2024-06-07T09:18:44.652024Z"
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>unique_id</th>\n",
              "      <th>first_name</th>\n",
              "      <th>surname</th>\n",
              "      <th>dob</th>\n",
              "      <th>city</th>\n",
              "      <th>email</th>\n",
              "      <th>cluster</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>922</th>\n",
              "      <td>922</td>\n",
              "      <td>Evie</td>\n",
              "      <td>Jones</td>\n",
              "      <td>2002-07-22</td>\n",
              "      <td>NaN</td>\n",
              "      <td>eviejones@brewer-sparks.org</td>\n",
              "      <td>230</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>224</td>\n",
              "      <td>Logn</td>\n",
              "      <td>Feeruson</td>\n",
              "      <td>2013-10-15</td>\n",
              "      <td>London</td>\n",
              "      <td>l.fergson46@shah.com</td>\n",
              "      <td>58</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     unique_id first_name   surname         dob    city  \\\n",
              "922        922       Evie     Jones  2002-07-22     NaN   \n",
              "224        224       Logn  Feeruson  2013-10-15  London   \n",
              "\n",
              "                           email  cluster  \n",
              "922  eviejones@brewer-sparks.org      230  \n",
              "224         l.fergson46@shah.com       58  "
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from splink import splink_datasets\n",
        "\n",
        "df = splink_datasets.fake_1000\n",
        "\n",
        "# Split a simple dataset into two, separate datasets which can be linked together.\n",
        "df_l = df.sample(frac=0.5)\n",
        "df_r = df.drop(df_l.index)\n",
        "\n",
        "df_l.head(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:44.695716Z",
          "iopub.status.busy": "2024-06-07T09:18:44.695390Z",
          "iopub.status.idle": "2024-06-07T09:18:44.942598Z",
          "shell.execute_reply": "2024-06-07T09:18:44.942052Z"
        }
      },
      "outputs": [],
      "source": [
        "import splink.comparison_library as cl\n",
        "\n",
        "from splink import DuckDBAPI, Linker, SettingsCreator, block_on\n",
        "\n",
        "settings = SettingsCreator(\n",
        "    link_type=\"link_only\",\n",
        "    blocking_rules_to_generate_predictions=[\n",
        "        block_on(\"first_name\"),\n",
        "        block_on(\"surname\"),\n",
        "    ],\n",
        "    comparisons=[\n",
        "        cl.NameComparison(\n",
        "            \"first_name\",\n",
        "        ),\n",
        "        cl.NameComparison(\"surname\"),\n",
        "        cl.DateOfBirthComparison(\n",
        "            \"dob\",\n",
        "            input_is_string=True,\n",
        "            invalid_dates_as_null=True,\n",
        "        ),\n",
        "        cl.ExactMatch(\"city\").configure(term_frequency_adjustments=True),\n",
        "        cl.EmailComparison(\"email\"),\n",
        "    ],\n",
        ")\n",
        "\n",
        "linker = Linker(\n",
        "    [df_l, df_r],\n",
        "    settings,\n",
        "    db_api=DuckDBAPI(),\n",
        "    input_table_aliases=[\"df_left\", \"df_right\"],\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:44.946395Z",
          "iopub.status.busy": "2024-06-07T09:18:44.946113Z",
          "iopub.status.idle": "2024-06-07T09:18:45.188705Z",
          "shell.execute_reply": "2024-06-07T09:18:45.188192Z"
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-61dfa5baef874167a95d6bc52b0c76e8.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-61dfa5baef874167a95d6bc52b0c76e8.vega-embed details,\n",
              "  #altair-viz-61dfa5baef874167a95d6bc52b0c76e8.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-61dfa5baef874167a95d6bc52b0c76e8\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-61dfa5baef874167a95d6bc52b0c76e8\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-61dfa5baef874167a95d6bc52b0c76e8\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"layer\": [{\"mark\": \"rect\", \"encoding\": {\"color\": {\"field\": \"completeness\", \"legend\": null, \"scale\": {\"scheme\": \"darkred\", \"zero\": true}, \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"source_dataset\", \"title\": \"Source dataset\", \"type\": \"nominal\"}, {\"field\": \"total_rows_inc_nulls\", \"format\": \",\", \"title\": \"# of records\", \"type\": \"quantitative\"}, {\"field\": \"column_name\", \"title\": \"Column name\", \"type\": \"nominal\"}, {\"field\": \"total_null_rows\", \"format\": \",\", \"title\": \"# of nulls\", \"type\": \"quantitative\"}, {\"field\": \"completeness\", \"format\": \".1%\", \"type\": \"quantitative\"}], \"x\": {\"axis\": {\"labelAngle\": 20}, \"field\": \"column_name\", \"sort\": {\"field\": \"mean_comp\", \"order\": \"descending\"}, \"title\": \"Column name\", \"type\": \"nominal\"}, \"y\": {\"field\": \"source_dataset\", \"title\": \"Source dataset\", \"type\": \"nominal\"}}, \"title\": \"Column completeness by source dataset\", \"transform\": [{\"joinaggregate\": [{\"op\": \"mean\", \"field\": \"completeness\", \"as\": \"mean_comp\"}], \"groupby\": [\"column_name\"]}]}, {\"mark\": {\"type\": \"text\"}, \"encoding\": {\"color\": {\"condition\": {\"test\": \"datum['completeness'] < 0.5\", \"value\": \"white\"}, \"value\": \"black\"}, \"text\": {\"field\": \"completeness\", \"format\": \".0%\", \"type\": \"quantitative\"}, \"x\": {\"axis\": {\"labelAngle\": 0}, \"field\": \"column_name\", \"sort\": {\"field\": \"mean_comp\", \"order\": \"descending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"source_dataset\", \"type\": \"nominal\"}}, \"transform\": [{\"joinaggregate\": [{\"op\": \"mean\", \"field\": \"completeness\", \"as\": \"mean_comp\"}], \"groupby\": [\"column_name\"]}]}], \"data\": {\"name\": \"data-464126c826d3483244c6f532dacf2289\"}, \"height\": {\"step\": 40}, \"width\": {\"step\": 40}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-464126c826d3483244c6f532dacf2289\": [{\"source_dataset\": \"df_left\", \"column_name\": \"first_name\", \"total_null_rows\": 83, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8339999914169312}, {\"source_dataset\": \"df_right\", \"column_name\": \"first_name\", \"total_null_rows\": 86, \"total_rows_inc_nulls\": 500, \"completeness\": 0.828000009059906}, {\"source_dataset\": \"df_right\", \"column_name\": \"surname\", \"total_null_rows\": 80, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8399999737739563}, {\"source_dataset\": \"df_left\", \"column_name\": \"surname\", \"total_null_rows\": 101, \"total_rows_inc_nulls\": 500, \"completeness\": 0.7979999780654907}, {\"source_dataset\": \"df_right\", \"column_name\": \"dob\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"df_left\", \"column_name\": \"dob\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"df_left\", \"column_name\": \"city\", \"total_null_rows\": 94, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8119999766349792}, {\"source_dataset\": \"df_right\", \"column_name\": \"city\", \"total_null_rows\": 93, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8140000104904175}, {\"source_dataset\": \"df_left\", \"column_name\": \"email\", \"total_null_rows\": 106, \"total_rows_inc_nulls\": 500, \"completeness\": 0.7879999876022339}, {\"source_dataset\": \"df_right\", \"column_name\": \"email\", \"total_null_rows\": 105, \"total_rows_inc_nulls\": 500, \"completeness\": 0.7900000214576721}]}}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.LayerChart(...)"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from splink.exploratory import completeness_chart\n",
        "\n",
        "completeness_chart(\n",
        "    [df_l, df_r],\n",
        "    cols=[\"first_name\", \"surname\", \"dob\", \"city\", \"email\"],\n",
        "    db_api=DuckDBAPI(),\n",
        "    table_names_for_chart=[\"df_left\", \"df_right\"],\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:45.192584Z",
          "iopub.status.busy": "2024-06-07T09:18:45.192253Z",
          "iopub.status.idle": "2024-06-07T09:18:45.341533Z",
          "shell.execute_reply": "2024-06-07T09:18:45.340965Z"
        }
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Probability two random records match is estimated to be  0.00338.\n",
            "This means that amongst all possible pairwise record comparisons, one in 295.61 are expected to match.  With 250,000 total possible comparisons, we expect a total of around 845.71 matching pairs\n"
          ]
        }
      ],
      "source": [
        "\n",
        "deterministic_rules = [\n",
        "    \"l.first_name = r.first_name and levenshtein(r.dob, l.dob) <= 1\",\n",
        "    \"l.surname = r.surname and levenshtein(r.dob, l.dob) <= 1\",\n",
        "    \"l.first_name = r.first_name and levenshtein(r.surname, l.surname) <= 2\",\n",
        "    block_on(\"email\"),\n",
        "]\n",
        "\n",
        "\n",
        "linker.training.estimate_probability_two_random_records_match(deterministic_rules, recall=0.7)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:45.344512Z",
          "iopub.status.busy": "2024-06-07T09:18:45.344289Z",
          "iopub.status.idle": "2024-06-07T09:18:46.142225Z",
          "shell.execute_reply": "2024-06-07T09:18:46.141712Z"
        }
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "You are using the default value for `max_pairs`, which may be too small and thus lead to inaccurate estimates for your model's u-parameters. Consider increasing to 1e8 or 1e9, which will result in more accurate estimates, but with a longer run time.\n",
            "----- Estimating u probabilities using random sampling -----\n",
            "\n",
            "Estimated u probabilities using random sampling\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (no m values are trained).\n",
            "    - city (no m values are trained).\n",
            "    - email (no m values are trained).\n"
          ]
        }
      ],
      "source": [
        "linker.training.estimate_u_using_random_sampling(max_pairs=1e6, seed=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:46.145662Z",
          "iopub.status.busy": "2024-06-07T09:18:46.145393Z",
          "iopub.status.idle": "2024-06-07T09:18:47.814138Z",
          "shell.execute_reply": "2024-06-07T09:18:47.813573Z"
        }
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "l.\"dob\" = r.\"dob\"\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - first_name\n",
            "    - surname\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - dob\n",
            "\n",
            "WARNING:\n",
            "Level Jaro-Winkler >0.88 on username on comparison email not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was -0.418 in the m_probability of surname, level `Exact match on surname`\n",
            "Iteration 2: Largest change in params was 0.104 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0711 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 4: Largest change in params was 0.0237 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.0093 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00407 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.0019 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.000916 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000449 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.000222 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.00011 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 5.46e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for email - Jaro-Winkler >0.88 on username (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - dob (no m values are trained).\n",
            "    - email (some m values are not trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "l.\"email\" = r.\"email\"\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - first_name\n",
            "    - surname\n",
            "    - dob\n",
            "    - city\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - email\n",
            "\n",
            "Iteration 1: Largest change in params was -0.483 in the m_probability of dob, level `Exact match on dob`\n",
            "Iteration 2: Largest change in params was 0.0905 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.02 in probability_two_random_records_match\n",
            "Iteration 4: Largest change in params was 0.00718 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.0031 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00148 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.000737 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.000377 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000196 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.000102 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 5.37e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 11 iterations\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - email (some m values are not trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "l.\"first_name\" = r.\"first_name\"\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - surname\n",
            "    - dob\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - first_name\n",
            "\n",
            "Iteration 1: Largest change in params was -0.169 in the m_probability of surname, level `All other comparisons`\n",
            "Iteration 2: Largest change in params was -0.0127 in the m_probability of surname, level `All other comparisons`\n",
            "Iteration 3: Largest change in params was -0.00388 in the m_probability of surname, level `All other comparisons`\n",
            "Iteration 4: Largest change in params was -0.00164 in the m_probability of email, level `Jaro-Winkler >0.88 on username`\n",
            "Iteration 5: Largest change in params was -0.00089 in the m_probability of email, level `Jaro-Winkler >0.88 on username`\n",
            "Iteration 6: Largest change in params was -0.000454 in the m_probability of email, level `Jaro-Winkler >0.88 on username`\n",
            "Iteration 7: Largest change in params was -0.000225 in the m_probability of email, level `Jaro-Winkler >0.88 on username`\n",
            "Iteration 8: Largest change in params was -0.00011 in the m_probability of email, level `Jaro-Winkler >0.88 on username`\n",
            "Iteration 9: Largest change in params was -5.31e-05 in the m_probability of email, level `Jaro-Winkler >0.88 on username`\n",
            "\n",
            "EM converged after 9 iterations\n",
            "\n",
            "Your model is fully trained. All comparisons have at least one estimate for their m and u values\n"
          ]
        }
      ],
      "source": [
        "session_dob = linker.training.estimate_parameters_using_expectation_maximisation(block_on(\"dob\"))\n",
        "session_email = linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    block_on(\"email\")\n",
        ")\n",
        "session_first_name = linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    block_on(\"first_name\")\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:47.817058Z",
          "iopub.status.busy": "2024-06-07T09:18:47.816828Z",
          "iopub.status.idle": "2024-06-07T09:18:48.064527Z",
          "shell.execute_reply": "2024-06-07T09:18:48.063844Z"
        }
      },
      "outputs": [],
      "source": [
        "results = linker.inference.predict(threshold_match_probability=0.9)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2024-06-07T09:18:48.067845Z",
          "iopub.status.busy": "2024-06-07T09:18:48.067582Z",
          "iopub.status.idle": "2024-06-07T09:18:48.084784Z",
          "shell.execute_reply": "2024-06-07T09:18:48.084179Z"
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>match_weight</th>\n",
              "      <th>match_probability</th>\n",
              "      <th>source_dataset_l</th>\n",
              "      <th>source_dataset_r</th>\n",
              "      <th>unique_id_l</th>\n",
              "      <th>unique_id_r</th>\n",
              "      <th>first_name_l</th>\n",
              "      <th>first_name_r</th>\n",
              "      <th>gamma_first_name</th>\n",
              "      <th>surname_l</th>\n",
              "      <th>...</th>\n",
              "      <th>dob_l</th>\n",
              "      <th>dob_r</th>\n",
              "      <th>gamma_dob</th>\n",
              "      <th>city_l</th>\n",
              "      <th>city_r</th>\n",
              "      <th>gamma_city</th>\n",
              "      <th>email_l</th>\n",
              "      <th>email_r</th>\n",
              "      <th>gamma_email</th>\n",
              "      <th>match_key</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3.180767</td>\n",
              "      <td>0.900674</td>\n",
              "      <td>df_left</td>\n",
              "      <td>df_right</td>\n",
              "      <td>242</td>\n",
              "      <td>240</td>\n",
              "      <td>Freya</td>\n",
              "      <td>Freya</td>\n",
              "      <td>4</td>\n",
              "      <td>Shah</td>\n",
              "      <td>...</td>\n",
              "      <td>1970-12-17</td>\n",
              "      <td>1970-12-16</td>\n",
              "      <td>4</td>\n",
              "      <td>Lonnod</td>\n",
              "      <td>noLdon</td>\n",
              "      <td>0</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3.180767</td>\n",
              "      <td>0.900674</td>\n",
              "      <td>df_left</td>\n",
              "      <td>df_right</td>\n",
              "      <td>241</td>\n",
              "      <td>240</td>\n",
              "      <td>Freya</td>\n",
              "      <td>Freya</td>\n",
              "      <td>4</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>1970-12-17</td>\n",
              "      <td>1970-12-16</td>\n",
              "      <td>4</td>\n",
              "      <td>London</td>\n",
              "      <td>noLdon</td>\n",
              "      <td>0</td>\n",
              "      <td>f.s@flynn.com</td>\n",
              "      <td>None</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3.212523</td>\n",
              "      <td>0.902626</td>\n",
              "      <td>df_left</td>\n",
              "      <td>df_right</td>\n",
              "      <td>679</td>\n",
              "      <td>682</td>\n",
              "      <td>Elizabeth</td>\n",
              "      <td>Elizabeth</td>\n",
              "      <td>4</td>\n",
              "      <td>Shaw</td>\n",
              "      <td>...</td>\n",
              "      <td>2006-04-21</td>\n",
              "      <td>2016-04-18</td>\n",
              "      <td>1</td>\n",
              "      <td>Cardiff</td>\n",
              "      <td>Cardifrf</td>\n",
              "      <td>0</td>\n",
              "      <td>e.shaw@smith-hall.biz</td>\n",
              "      <td>e.shaw@smith-hall.lbiz</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3.224126</td>\n",
              "      <td>0.903331</td>\n",
              "      <td>df_left</td>\n",
              "      <td>df_right</td>\n",
              "      <td>576</td>\n",
              "      <td>580</td>\n",
              "      <td>Jessica</td>\n",
              "      <td>Jessica</td>\n",
              "      <td>4</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>1974-11-17</td>\n",
              "      <td>1974-12-17</td>\n",
              "      <td>4</td>\n",
              "      <td>None</td>\n",
              "      <td>Walsall</td>\n",
              "      <td>-1</td>\n",
              "      <td>jesscac.owen@elliott.org</td>\n",
              "      <td>None</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3.224126</td>\n",
              "      <td>0.903331</td>\n",
              "      <td>df_left</td>\n",
              "      <td>df_right</td>\n",
              "      <td>577</td>\n",
              "      <td>580</td>\n",
              "      <td>Jessica</td>\n",
              "      <td>Jessica</td>\n",
              "      <td>4</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>1974-11-17</td>\n",
              "      <td>1974-12-17</td>\n",
              "      <td>4</td>\n",
              "      <td>None</td>\n",
              "      <td>Walsall</td>\n",
              "      <td>-1</td>\n",
              "      <td>jessica.owen@elliott.org</td>\n",
              "      <td>None</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 22 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   match_weight  match_probability source_dataset_l source_dataset_r  \\\n",
              "0      3.180767           0.900674          df_left         df_right   \n",
              "1      3.180767           0.900674          df_left         df_right   \n",
              "2      3.212523           0.902626          df_left         df_right   \n",
              "3      3.224126           0.903331          df_left         df_right   \n",
              "4      3.224126           0.903331          df_left         df_right   \n",
              "\n",
              "   unique_id_l  unique_id_r first_name_l first_name_r  gamma_first_name  \\\n",
              "0          242          240        Freya        Freya                 4   \n",
              "1          241          240        Freya        Freya                 4   \n",
              "2          679          682    Elizabeth    Elizabeth                 4   \n",
              "3          576          580      Jessica      Jessica                 4   \n",
              "4          577          580      Jessica      Jessica                 4   \n",
              "\n",
              "  surname_l  ...       dob_l       dob_r gamma_dob   city_l    city_r  \\\n",
              "0      Shah  ...  1970-12-17  1970-12-16         4   Lonnod    noLdon   \n",
              "1      None  ...  1970-12-17  1970-12-16         4   London    noLdon   \n",
              "2      Shaw  ...  2006-04-21  2016-04-18         1  Cardiff  Cardifrf   \n",
              "3      None  ...  1974-11-17  1974-12-17         4     None   Walsall   \n",
              "4      None  ...  1974-11-17  1974-12-17         4     None   Walsall   \n",
              "\n",
              "  gamma_city                   email_l                 email_r gamma_email  \\\n",
              "0          0                      None                    None          -1   \n",
              "1          0             f.s@flynn.com                    None          -1   \n",
              "2          0     e.shaw@smith-hall.biz  e.shaw@smith-hall.lbiz           3   \n",
              "3         -1  jesscac.owen@elliott.org                    None          -1   \n",
              "4         -1  jessica.owen@elliott.org                    None          -1   \n",
              "\n",
              "  match_key  \n",
              "0         0  \n",
              "1         0  \n",
              "2         0  \n",
              "3         0  \n",
              "4         0  \n",
              "\n",
              "[5 rows x 22 columns]"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results.as_pandas_dataframe(limit=5)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}